# -*- coding: utf-8 -*-
from scrapy.linkextractors import LinkExtractor
# from girl.items import city
from ..items import city
import re
from scrapy.spiders import Rule
from datetime import *
from pyquery import PyQuery as pq
from scrapy.http import Request as sreq
from girl.items.city import *
import platform
from scrapy_redis.spiders import RedisSpider
import redis

class NewstorecitySpider(RedisSpider):
    name = "newcity"
    start_urls = [
        'https://www.cityheaven.net/hokkaido/shop-list/ns-renewal/',
    ]
    custom_settings = {
        # "DEPTH_PRIORITY" : 1,
        "CONCURRENT_REQUESTS": 550,
        "HTTPCACHE_EXPIRATION_SECS": 60 * 60 * 34 * 3
    }
    pool = redis.ConnectionPool(host='localhost', port=6379)
    conn = redis.Redis(connection_pool=pool)
    conn.sadd("newcity:start_urls", *start_urls)
    if platform.system() != 'Darwin':
        time = datetime.now().strftime('%Y-%m-%d-%H-%M')
        custom_settings["LOG_FILE"] = "/mnt/scrapy/crawler/%s-%s.log" % (name, time)
    stats = {}

    def __init__(self, *a, **k):
        super(NewstorecitySpider, self).__init__(*a, **k)
        if k.get("type"):
            self.start_urls = ['https://www.cityheaven.net/hokkaido/shop-list/ns-renewal/']
            # self.conn.sadd("city:start_urls", *self.start_urls)

    def parse(self, response):
        urls = response.xpath("//ul[@class='areaSelectionList clearfix']//a/@href").extract()
        for x in urls:
            yield sreq(response.urljoin(x), callback=self.parse0,dont_filter=True)

    def parse0(self, response):
        item = city.girlItem()
        head = {"User-Agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1"}
        doc = pq(response.text)
        for x in doc(".shop_list.shop_list_s"):
            url = doc(x)("a.shop_title_shop").attr("href")
            info = doc(x)(".base_info dd").text()
            price = doc(x)(".coupon_price").text()
            name = doc(x)(".shop_title_shop span").text()
            url = url.replace("---", "")
            tmp = doc(x)(".shop_title_gyousyu").text()
            cate1 = tmp and tmp.split(u"\uff08")[0] or ""
            cate2 = tmp.split(u"\uff08")[1] and tmp.split(u"\uff08")[1].split("/")[0] or ""
            yield sreq(response.urljoin(url), callback=self.store,
                meta={'info': info, "price": price, "name": name, "cate1": cate1, "cate2": cate2})
            yield sreq(response.urljoin(url), callback=self.parse1,headers=head)

        for x in doc(".shop_list.shop_list_a").items():
            url = x("a.shop_title_shop").attr("href")
            info = x(".base_info dd").text()
            price = x(".coupon_price").text()
            name = x(".shop_title_shop span").text()
            url = url.replace("---", "")
            tmp = x(".shop_title_gyousyu").text()
            cate1 = tmp and tmp.split(u"\uff08")[0] or ""
            cate2 = tmp.split(u"\uff08")[1] and tmp.split(u"\uff08")[1].split("/")[0] or ""
            yield sreq(response.urljoin(url), callback=self.store,
                       meta={'info': info, "price": price, "name": name, "cate1": cate1, "cate2": cate2})
            yield sreq(response.urljoin(url), callback=self.parse1,headers=head)

        for x in doc(".shop_list.shop_list_d").items():
            url = x("a.shop_title_shop").attr("href")
            info = x(".base_info").text()
            tmp = x(".shop_title_gyousyu").text()
            name = x(".shop_title_shop span").text()
            cate1 = tmp and tmp.split(u"\uff08")[0] or ""
            cate2 = tmp.split(u"\uff08")[1] and tmp.split(u"\uff08")[1].split("/")[0] or ""
            price = x(".base_info dd:eq(1)").text()
            url = url.replace("---", "")
            yield sreq(response.urljoin(url), callback=self.store2,
                       meta={'info': info, "price": price, "cate1": cate1, "cate2": cate2, "name": name})
            yield sreq(response.urljoin(url), callback=self.parse1,headers=head)

        for x in doc("li[class='shop_list ']").items():
            url = x("a.shop_title_shop").attr("href")
            info = x(".base_info dd").text()
            name = x(".shop_title_shop span").text()
            workTime = x(".base_info dd:eq(0)").text()
            price = x(".coupon_price").text() or x(".base_info dd:eq(1)").text()
            address = x(".base_info dd:eq(2)").text()
            tmp = x(".shop_title_gyousyu").text()
            cate1 = tmp and tmp.split(u"\uff08")[0] or ""
            cate2 = tmp.split(u"\uff08")[1] and tmp.split(u"\uff08")[1].split("/")[0] or ""
            url = url.replace("---", "")
            yield sreq(response.urljoin(url), callback=self.store3,
                       meta={'info': info, "price": price, "cate1": cate1, "cate2": cate2,
                             "address": address, "workTime": workTime, "name": name})
            yield sreq(response.urljoin(url), callback=self.parse1,headers=head)
        url = response.css(".shop_nav_list .next::attr(href)").extract_first()
        if url:
            if re.search("/\d/",url):
                num = re.search("/(\d)/",url).group(1)
                if num < 4:
                    yield sreq(response.urljoin(url), callback=self.parse0,dont_filter=True)


    def parse1(self, response):
        item = smallItem()
        item["url"] = response.url
        if response.css(".logo-nodeco::attr(data-echo)"):
            item["smallcover"] = response.css(".logo-nodeco::attr(data-echo)").re_first("(img.*)\?cache")
        elif response.css(".shopinfo img::attr(src)"):
            item["smallcover"] = response.css(".shopinfo img::attr(src)").re_first("(img.*)\?cache")
        elif response.css(".shoptopimage img::attr(data-echo)"):
            item["smallcover"] = response.css(".shoptopimage img::attr(data-echo)").re_first("(img.*)\?cache")
        else:
            item["smallcover"] = "no"
        yield item

    def girlList2(self, response):
        for x in response.xpath("//ul[@id='girl_list']/li"):
            url = x.xpath("div/a/@href").extract_first()
            name = "".join(x.css("a::text").extract()).strip()
            print url
            yield sreq(response.urljoin(url), callback=self.girl, priority=-1,meta={"name":name})
        next = response.xpath("//*[@class='next']/@href").extract_first()
        if next:
            if re.search("/\d/",next):
                num = re.search("/(\d)/",next).group(1)
                if num < 34:
                    yield sreq(response.urljoin(next), callback=self.girlList2,priority=-1,dont_filter=True)

    def girlList(self, response):
        doc = pq(response.body)
        for x in doc(".girllistimg").items():
            # img = x("img").attr("href")
            url = x("a").attr("href")
            print url
            yield sreq(response.urljoin(url), callback=self.girl, priority=-1)
        next = response.xpath("//*[@class='next']/@href").extract_first()
        if next:
            if re.search("/\d/",next):
                num = re.search("/(\d)/",next).group(1)
                if num < 34:
                    yield sreq(response.urljoin(next), callback=self.girlList,priority=-1,dont_filter=True)

    def mapImg(self, image_urls):
        if image_urls[0]:
            image_urls = map(lambda url: re.sub("(.*)img\.", "https://", url), image_urls)
            image_urls = map(lambda url: re.sub("//d-markets", "https://d-markets", url), image_urls)
            image_urls = map(lambda url: re.sub("\?cache.*", "", url), image_urls)
            image_urls = [x for x in image_urls if len(x) > 2]
        return image_urls

    def diary(self, response):
        tmp = []
        doc = pq(response.body)
        item = diaryItem()
        for x in doc(".diary_photolay_tbl").items():
            time = x(".diarytime").text()
            name = x(".diarytime").next("a").text()
            girlUrl = x(".diarytime").next("a").attr("href")
            title = x(".diary_title").text()
            img = x(".diary_photoimage_tbl img").attr("src")
            context = x("p:eq(1)").text()
            item["time"] = time
            item["name"] = name
            item["girlUrl"] = girlUrl
            item["title"] = title
            item["url"] = x(".diary_photoimage_tbl a").attr("href")
            # item["image_urls"] = ""
            item["date"] = str(date.today())
            if img:
                img = re.sub("(.*)img\.", "https://", img)
                img = re.sub("//d-markets", "https://d-markets", img)
                item["image_urls"] = img
            item["context"] = context
            yield item
            next = response.xpath("//*[@class='next']/@href").extract_first()
            if next:
                if re.search("/\d/",next):
                    num = re.search("/(\d)/",next).group(1)
                    if num < 44:
                        yield sreq(response.urljoin(next), callback=self.diary,dont_filter=True)

    def parseEmail(self, response):
        email = ""
        if response.css(".telmail a::text").extract_first():
            email = response.css(".telmail a::text").extract_first().lstrip("MAIL:")
        if not email:
            email = re.search(r"\w+@\S+\w", response.body).group()
        if not email:
            import ipdb;

            ipdb.set_trace()
        item = emailItem()
        url = response.meta["url"]
        item["url"] = response.url
        item["storesUrl"] = url
        item["email"] = email
        yield item

    def store2(self, response):
        item = storesItem()
        item["phone"] = response.css(".phone::text").extract_first() or response.xpath(
            "//p[@itemprop='telephone']//text()").extract_first()
        try:
            if len(item["phone"]) < 3:
                item["phone"] = response.css(".number::text").extract()[-1]
        except:
            item["phone"] = ""
        # item["name"] = response.xpath("//li/span[@itemprop='name']/text()").extract_first()
        item["name"] = response.meta.get("name")
        item["info"] = response.css(".introduction_info::text").extract_first()
        item["workTime"] = response.css(".colspan2::text").extract_first()
        item["address"] = response.css(".adr::text").extract_first()
        item["price"] = response.meta.get("price") or response.css(".pricerange::text").extract_first()
        item["cate1"] = response.meta.get("cate1")
        item["cate2"] = response.meta.get("cate2")
        tmp = [x.css("span::text").extract_first() for x in response.css("#location-breadcrumbs-wrap li")]
        area = [x for x in tmp if x not in ["/", None]]
        item["area"] = area
        item["url"] = response.url
        item["type"] = 3
        item["date"] = str(date.today())
        yield item

    def store3(self, response):
        item = storesItem()
        item["site"] = response.xpath("//a[@itemprop='sameAs']/text()").extract_first()
        item["phone"] = response.xpath("//span[@itemprop='telephone']/text()").extract_first()
        if not item.get("phone"):
            item["phone"] = response.css(".number::text").extract[-1]
        item["name"] = response.meta.get("name")
        item["info"] = response.xpath("//span[@itemprop='description']/text()").extract_first()
        item["workTime"] = response.meta.get("workTime")
        item["address"] = response.meta.get("address")
        item["price"] = response.meta.get("price")
        item["cate1"] = response.meta.get("cate1")
        item["cate2"] = response.meta.get("cate2")
        tmp = [x.css("span::text").extract_first() for x in response.css("#location-breadcrumbs-wrap li")]
        area = [x for x in tmp if x not in ["/", None]]
        item["area"] = area
        item["url"] = response.url
        item["date"] = str(date.today())
        item["cover"] = response.css("#shoplogo img ::attr(src)").re("(img.*)\?")
        if not item["cover"]:
            item["cover"] = response.css("#fakeLoader img ::attr(src)").re("(img.*)\?")
        item["type"] = 2
        item["listinfo"] = response.css(".info_text2::text").extract()
        yield item
        # url = response.css(".shop_menu_girls_recruit_display_toggle a::attr(href)").extract_first()
        # yield sreq(url,callback=self.parseEmail,meta={"url":response.url})
        yield sreq(response.urljoin("diarylist"), callback=self.diary)
        yield sreq(response.urljoin("girllist"), callback=self.girlList2)
        yield sreq(response.urljoin("shopmovie"), callback=self.movieList)

    def store(self, response):
        doc = pq(response.body)
        item = storesItem()
        image_urls = ""
        image_urls = [x.attr("src") for x in doc(".firstview-slider").items()]
        if not image_urls:
            tmp = re.findall("image_file_name\S+\"(\S+)\"?\,", response.body)
            if tmp:
                image_urls = tmp[0].replace("\\", "")
            else:
                pass
        if image_urls:
            image_urls = self.mapImg(image_urls)
        if not image_urls:
            image_urls = response.css("#one_slide ::attr('style')").extract_first()
            if image_urls:
                image_urls = re.findall("img.*\.[jpeg,png,jpg]{2,4}", image_urls)
        phone = doc(".fa.fa-lg.fa-phone").parent("p").text()
        if not phone:
            phone = response.xpath("//td[@itemprop='telephone']/text()").extract_first()
        workTime = doc(".fa.fa-lg.fa-clock-o").parent("p").text()
        if not workTime:
            workTime = response.xpath("//td[@class='shoptitle_com_item']/text()").extract()
        goto = doc(".fa.fa-lg.fa-street-view").parent("p").text()
        if not goto:
            goto = response.xpath("//td[@class='shoptitle_com_item']/text()").extract()
        # name = response.xpath("//li/span[@itemprop='name']/text()").extract_first()
        name = response.meta.get("name")
        tmp = [x("a").text() for x in doc("#location-breadcrumbs-wrap li").items()]
        area = [x for x in tmp if x not in ["/", None]]
        address = doc(".info_text2").html()
        try:
            address = re.search(r"</i>(.*)\[", address).groups()[-1]
        except Exception, e:
            pass
        tmp = response.css("#shoptitle_information p:nth-child(2) span::text").extract_first()
        item["cate2"] = response.meta.get("cate2")
        item["cate1"] = response.meta.get("cate1")
        site = doc("#info_site li").text()
        site = re.findall("http://\S+", site)
        # https://www.cityheaven.net/fukuoka/A4001/A400101/midarana_ol 参考这个地址
        info = doc(".shoptitle_block").text()
        if not address and not workTime and not phone:
            phone = doc(".number:eq(0)").text()
            workTime = doc(".tableinfo.shop-tableinfo dd:eq(0)").text()
            address = doc(".tableinfo.shop-tableinfo dd:eq(2)").text()
            site = doc(".tableinfo.shop-tableinfo dd:eq(3)").text()
            info = doc(".store_subtitle").text()
        if not address and not workTime and not phone:
            for x in response.xpath("//*[@class='shoptitle_com_info_detail']//td/text()").extract():
                if re.match("\d+-\d+-\s+", x):
                    phone = x
                elif x.find("~"):
                    workTime = x
            address1 = response.xpath("//*[@id='location-breadcrumbs-wrap']//span/text()")[-2].extract()
            address = response.xpath("//*[@id='location-breadcrumbs-wrap']//span/text()")[-2].extract() + address1
            site = response.xpath("//*[@id='info_site']//a/@href").extract()
        if not address and not workTime and not phone:
            import ipdb;

            ipdb.set_trace()
        attendance = []
        cost = ""
        url = doc(".shop_menu_girls_recruit_display_toggle a").attr("href")
        # phone = phone or response.css(".number::text").extract_first()
        # if not phone:
        #     phone = response.css(".number::text").extract()[-1]
        if not phone:
            tmp = re.search("\d+-\d+-\d+", response.body)
            if tmp:
                phone = tmp.group()

        if url and url.find("http:") < 0:
            url = "http:" + url
        if not url:
            try:
                url = response.xpath("//*[@id='shopbody']/div[1]/div/ul/li[3]/a/@href").extract()[1]
                url = "https://www.cityheaven.net" + url
            except Exception, e:
                # import ipdb;ipdb.set_trace()
                url = ""
                return
        emailUrl = url
        item["cover"] = response.css(".firstview-slider ::attr(src)").re("(img.*)\?")
        if not item["cover"]:
            item["cover"] = response.css("#fakeLoader img ::attr(src)").re("(img.*)\?")
        item["image_urls"] = image_urls
        item["name"] = name
        item["address"] = address
        item["goto"] = goto
        item["workTime"] = workTime
        item["site"] = site
        item["info"] = info or "info"
        item["area"] = area
        item["phone"] = phone
        item["listinfo"] = response.meta.get("info")
        item["price"] = response.meta.get("price")
        item["attendance"] = attendance
        item["date"] = str(date.today())
        item["url"] = response.url
        item["type"] = 1
        yield item
        yield sreq(emailUrl, callback=self.parseEmail, meta={"url": item["url"]})
        url = response.urljoin("diarylist")
        yield sreq(url, callback=self.diary)
        url = response.urljoin("girllist")
        yield sreq(url, callback=self.girlList)
        url = response.urljoin("shopmovie")
        yield sreq(url, callback=self.movieList)

    def girl(self, response):
        item = girlItem()
        doc = pq(response.body)
        try:
            name = response.xpath('//tr/td[@itemprop="name"]/text()').extract()[0]
        except Exception, e:
            name = response.xpath('//div[@id="profiles"]/h4/text()').extract_first()

        tmp = ["意見箱にお答えします", "新人日給", "体験入店予定", "LINEで簡単お問い合わせ"]
        if type(name) == unicode:
            name = name.encode("utf-8")
        if not name:
            return
        try:
            tmps = map(name.find, tmp)
        except:
            import ipdb;

            ipdb.set_trace()
        for x in tmps:
            if x >= 0:
                # import ipdb;ipdb.set_trace()
                return
        # age = response.xpath("//*[@id='p_data']/tr[2]/td/text()").extract()[0]
        # birthday = response.xpath("//*[@id='p_data']/tr[3]/td/text()").extract()[0]
        bwh = ""
        constellation = ""
        blood = ""
        age = ""
        other = response.xpath("//*[@id='p_data']//td/text()").extract()
        for x in other:
            if x.find("T") >= 0:
                bwh = x
            elif x.find("歳") > 0:
                age = x
            elif x.find("座") > 0:
                constellation = x
            elif x.find("型") > 0:
                blood = x
        imgList = []
        imgList = response.xpath("//ul[@id='slider']//img/@src").re("(img.*)\?")
        if not imgList:
            imgList = response.css(".profile_image img::attr(src)").re("(img.*)\??")
        imgList = imgList + response.css("#thum img::attr(src)").re("(img.*)\??")
        if imgList:
            imgList = self.mapImg(imgList)
        questions = []
        for x in doc("#profile_question dl").items():
            questions.append(x.text())
        storeComment = doc(".commentmanager").text()
        comment = doc(".commentmanager").text()
        character = []
        for x in doc(".salespoint_list:eq(0) li").items():
            character.append(x.text())
        # storesUrl = self.getSite(doc, response)
        point = response.xpath("//*[@class='salespoint_list']/li/text()").extract()
        # import ipdb;ipdb.set_trace()
        item["name"] = response.meta.get("name") or name
        item["age"] = age
        item["bwh"] = bwh
        item["blood"] = blood
        item["constellation"] = constellation
        item["storesUrl"] = re.sub("girlid-.*","",response.url)
        item["url"] = response.url
        item["questions"] = ''
        item["comment"] = ''
        item["storeComment"] = ''
        item["other"] = other
        item["character"] = character
        item["point"] = point
        item["image_urls"] = imgList
        item["date"] = str(date.today())
        yield item

    def movieList(self, response):
        item = movieItem()
        doc = pq(response.body)
        for x in doc(".shop_movie_data").items():
            url = x("a:last").attr("href")
            if not url:
                break
            thumb = x(".movieThumb img").attr("src")
            girlUrl = x(".movieName a").attr("href")
            item["thumb"] = thumb
            item["girlUrl"] = girlUrl
            yield sreq(response.urljoin(url), callback=self.getMovie, meta={"item": item})
        next = response.xpath("//*[@class='next']/@href").extract_first()
        if next:
            if re.search("/\d/",next):
                num = re.search("/(\d)/",next).group(1)
                if num < 34:
                    yield sreq(response.urljoin(next), callback=self.movieList,dont_filter=True)

    def getMovie(self, response):
        item = movieItem()
        movie = response.css("#videofile::attr(src)").extract_first()
        item = response.meta["item"]
        item["movie"] = movie
        item["url"] = response.url
        item["date"] = str(date.today())
        yield item